package singleThreaded;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * PageRetriever: retrieves the source of the url.
 * @author zsyed
 *
 */
public class PageRetriever {
    /** 
     * list containing all the urls
     */
    private List<String> my_string_urls;
    
    /** 
     * list containing all the documents 
     */ 
    private List<URL> my_source_urls;
    
    /** 
     * the max number of pages crawl 
     */
    private int my_crawl_limit;
    
    /** 
     * a parser. 
     */
    private PageParserSingle my_parser;
    
    /**
     * constructing the page retriever.
     * @param the_crawl_limit the limit of pages to be retrieve.
     */
    public PageRetriever(int the_crawl_limit) {
        my_crawl_limit = the_crawl_limit;
        my_parser = new PageParserSingle();
        my_source_urls = new ArrayList<URL>();
        my_string_urls = my_parser.getLinks();  
    }
    
    /**
     *  crawl the web starting from the start url.
     */
    public void crawlPages(String the_start) throws IOException {
        URL source = null;
        my_string_urls.add(the_start);
        int i = 0;
        while (i < my_string_urls.size()) {
            try {
                source = new URL(my_string_urls.get(i).toString());
            } catch (Exception e1) {
            }
            if (source != null && source.toString().contains("http")||
                    my_string_urls.get(i).toString().contains("txt")){ // AS changed "html" to "http"
                my_parser.parse(source);
                my_source_urls.add(source);
                //determine how many web pages crawl
                if (my_source_urls.size() == my_crawl_limit) {
                    break;  
                }
                i++;
            } else {
                //Only happen when it's not html or txt. skip
                i++;
            }
        }
    }
    /**
     *  get the list of urls. 
     */ 
    public List<String> getStringUrls() {
        return my_string_urls;
    }
    
    /**
     *  get the list of source containing the contents of each urls. 
     */
    public List<URL> getURLs() {
        return my_source_urls;
    }
    
    /**
     *  getting the current parser 
     */
    public PageParserSingle getCurrentParser() {
        return my_parser;
    }
}